This is a Project about skin cancer detection using AI¶

First we import all the libraries we will be using¶

In [1]:
import os
import cv2
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torch.optim as optim
import torchvision.models as models
import time
import itertools


from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
from torch.utils.data import DataLoader, Dataset
from torchvision import models
from PIL import Image
from torchvision.models import ResNet50_Weights
from sklearn.model_selection import train_test_split
from torchvision import transforms
from pytorch_grad_cam import GradCAM
from pytorch_grad_cam.utils.image import preprocess_image
from sklearn.utils.class_weight import compute_class_weight
from torch.optim.lr_scheduler import StepLR
import torch.optim.lr_scheduler as lr_scheduler

We will be loading our dataset which contain about 10,000 images from ISIC(HAM10000)¶

In [2]:
# Define paths
metadata_path = r"E:\afeka\finalProject\JupyterLab\ISIC-images\metadata.csv"
root_dir = r"E:\afeka\finalProject\JupyterLab\ISIC-images"

# Load metadata
metadata = pd.read_csv(metadata_path)

# Fix image filenames (append .jpg)
metadata["image_name"] = metadata["isic_id"] + ".jpg"

# Check if images exist
missing_files = [f for f in metadata["image_name"] if not os.path.exists(os.path.join(root_dir, f))]

if missing_files:
    print("Missing files:", missing_files[:5])  # Print a sample
else:
    print("All images exist.")

All images exist.
In [3]:
print(metadata.head())

        isic_id                                        attribution  \
0  ISIC_0024306  ViDIR Group, Department of Dermatology, Medica...   
1  ISIC_0024307  ViDIR Group, Department of Dermatology, Medica...   
2  ISIC_0024308  ViDIR Group, Department of Dermatology, Medica...   
3  ISIC_0024309  ViDIR Group, Department of Dermatology, Medica...   
4  ISIC_0024310  ViDIR Group, Department of Dermatology, Medica...   

  copyright_license  age_approx anatom_site_general anatom_site_special  \
0          CC-BY-NC        45.0                 NaN                 NaN   
1          CC-BY-NC        50.0     lower extremity                 NaN   
2          CC-BY-NC        55.0                 NaN                 NaN   
3          CC-BY-NC        40.0                 NaN                 NaN   
4          CC-BY-NC        60.0      anterior torso                 NaN   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   image_type   lesion_id  melanocytic  \
0  serial imaging showing no change  dermoscopic  IL_7252831         True   
1  serial imaging showing no change  dermoscopic  IL_6125741         True   
2  serial imaging showing no change  dermoscopic  IL_3692653         True   
3  serial imaging showing no change  dermoscopic  IL_0959663         True   
4                    histopathology  dermoscopic  IL_8194852         True   

      sex        image_name  
0    male  ISIC_0024306.jpg  
1    male  ISIC_0024307.jpg  
2  female  ISIC_0024308.jpg  
3    male  ISIC_0024309.jpg  
4    male  ISIC_0024310.jpg

now we will print all the labels and the count for the label "diagnosis"¶

In [4]:
print(metadata.columns.tolist())  # List all column names
print(metadata["diagnosis"].value_counts())  # Check label distribution

['isic_id', 'attribution', 'copyright_license', 'age_approx', 'anatom_site_general', 'anatom_site_special', 'benign_malignant', 'concomitant_biopsy', 'diagnosis', 'diagnosis_1', 'diagnosis_2', 'diagnosis_3', 'diagnosis_confirm_type', 'image_type', 'lesion_id', 'melanocytic', 'sex', 'image_name']
diagnosis
nevus                         7737
pigmented benign keratosis    1338
melanoma                      1305
basal cell carcinoma           622
squamous cell carcinoma        229
vascular lesion                180
dermatofibroma                 160
actinic keratosis              149
Name: count, dtype: int64

Mapping Lesions to Numeric Labels¶

In [5]:
# Define the label mapping function
def map_labels(row):
    # Nevus mapping
    if row["diagnosis"] == "nevus":
        return 0  # Nevus
    
    # Melanoma mapping
    elif row["diagnosis"] == "melanoma":
        return 1  # Melanoma
    
    # If diagnosis is from other specific types and diagnosis_1 is Benign, classify as Nevus
    elif (row["diagnosis"] in ["pigmented benign keratosis", "basal cell carcinoma", "squamous cell carcinoma", 
                               "vascular lesion", "dermatofibroma", "actinic keratosis"]) and row["diagnosis_1"] == "Benign":
        return 0  # Nevus
    
    # For Other lesion (Malignant or Indeterminate)
    elif row["diagnosis"] == "Other lesion":
        if row["diagnosis_1"] in ["Malignant", "Indeterminate"]:
            return 2  # Other lesion (Malignant or Indeterminate)
    
    # If diagnosis is one of the other types and diagnosis_1 is Malignant or Indeterminate, classify as Other lesion
    if row["diagnosis_1"] in ["Malignant", "Indeterminate"]:
        return 2  # Other lesion

    return 2  # Default to Other lesion for any other case

# Apply the mapping function to create a new column 'mapped_label'
metadata["mapped_label"] = metadata.apply(map_labels, axis=1)

# Print class distribution
class_distribution = metadata["mapped_label"].value_counts().sort_index()
class_labels = {
    0: "Nevus",
    1: "Melanoma",
    2: "Other lesion"
}

# Output the class distribution
print("Class Distribution:")
for label, count in class_distribution.items():
    print(f"{class_labels[label]}: {count}")

Class Distribution:
Nevus: 9415
Melanoma: 1305
Other lesion: 1000

Creating and Loading the ISICDataset for Skin Lesion Classification¶

In [6]:
class ISICDataset(Dataset):
    def __init__(self, metadata_path, root_dir, transform=None):
        """
        Args:
            metadata_path (str): Path to the CSV file containing metadata.
            root_dir (str): Directory with all the images.
            transform (callable, optional): Transformations to be applied to images.
        """
        self.metadata = pd.read_csv(metadata_path)
        self.metadata["image_name"] = self.metadata["isic_id"] + ".jpg"  # Fix filenames
        self.root_dir = root_dir
        self.transform = transform

        # Apply the label mapping logic and add the 'mapped_label' column
        self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1)

    def __len__(self):
        return len(self.metadata)

    def __getitem__(self, idx):
        img_name = self.metadata.iloc[idx]["image_name"]
        label = self.metadata.iloc[idx]["mapped_label"]  # Use 'mapped_label' instead of the old 'label'

        img_path = os.path.join(self.root_dir, img_name)
        
        # Open image
        image = Image.open(img_path).convert("RGB")  # Ensure 3 channels
        
        # Apply transformations
        if self.transform:
            image = self.transform(image)
        
        return image, label

# Define transformations (data augmentation)
transform = transforms.Compose([
    transforms.RandomHorizontalFlip(p=0.5),  # Flip 50% of images horizontally
    transforms.RandomRotation(degrees=15),   # Rotate images randomly by ±15 degrees
    transforms.RandomResizedCrop(224, scale=(0.8, 1.0)),  # Random crop + resize
    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Adjust colors
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
 ])

"""
transform = transforms.Compose([
    transforms.Resize((224, 224)),  # Resize images to 224x224
    transforms.RandomRotation(10),  # Randomly rotate images in the range (-10 to 10 degrees)
    transforms.RandomAffine(degrees=0, scale=(0.9, 1.1)),  # Randomly zoom image, scale range between 90% to 110%
    transforms.RandomHorizontalFlip(),  # Randomly flip images horizontally
    transforms.RandomVerticalFlip(),  # Randomly flip images vertically
    transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)),  # Randomly shift images horizontally and vertically
    transforms.ToTensor(),  # Convert image to tensor
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize with ImageNet mean/std
])
"""
dataset = ISICDataset(metadata_path, root_dir, transform=transform)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

# Test if the dataloader works
for images, labels in dataloader:
    print(f"Batch of images: {images.shape}, Labels: {labels.shape}")
    break  # Check one batch

Batch of images: torch.Size([32, 3, 224, 224]), Labels: torch.Size([32])

Visualizing the Distribution of Lesion Types¶

In [7]:
# Define the label mapping
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}

# Map the numeric labels to the string labels for better visualization
mapped_label_counts = dataset.metadata["mapped_label"].map(label_mapping).value_counts()

# Create the bar chart
plt.figure(figsize=(6, 4))
bars = plt.bar(mapped_label_counts.index, mapped_label_counts.values, color=['blue', 'gray', 'red'])

# Add count labels on top of each bar
for bar in bars:
    height = bar.get_height()
    plt.text(bar.get_x() + bar.get_width() / 2, height, f'{height}', ha='center', va='bottom', fontsize=10)

# Add labels and title
plt.xlabel("Lesion Type")
plt.ylabel("Count")
plt.title("Distribution of Lesion Types")
plt.xticks(ticks=range(len(mapped_label_counts.index)), labels=mapped_label_counts.index, rotation=45)
plt.show()

# Pie chart
plt.figure(figsize=(6, 6))
plt.pie(mapped_label_counts.values, labels=mapped_label_counts.index, autopct='%1.1f%%', colors=['blue', 'gray', 'red'])
plt.title("Proportion of Lesion Types")
plt.show()
No description has been provided for this image
No description has been provided for this image

Data Preprocessing: Removing Unnecessary Columns and Handling Missing Values¶

In [8]:
columns_to_remove = ['attribution', 'copyright_license', 'image_type']  
metadata = metadata.drop(columns=columns_to_remove)  

# Display the first few rows to confirm
print(metadata.head())

        isic_id  age_approx anatom_site_general anatom_site_special  \
0  ISIC_0024306        45.0                 NaN                 NaN   
1  ISIC_0024307        50.0     lower extremity                 NaN   
2  ISIC_0024308        55.0                 NaN                 NaN   
3  ISIC_0024309        40.0                 NaN                 NaN   
4  ISIC_0024310        60.0      anterior torso                 NaN   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   lesion_id  melanocytic     sex  \
0  serial imaging showing no change  IL_7252831         True    male   
1  serial imaging showing no change  IL_6125741         True    male   
2  serial imaging showing no change  IL_3692653         True  female   
3  serial imaging showing no change  IL_0959663         True    male   
4                    histopathology  IL_8194852         True    male   

         image_name  mapped_label  
0  ISIC_0024306.jpg             0  
1  ISIC_0024307.jpg             0  
2  ISIC_0024308.jpg             0  
3  ISIC_0024309.jpg             0  
4  ISIC_0024310.jpg             1
In [9]:
# Check if any missing values remain
print(metadata.isnull().sum())

isic_id                       0
age_approx                  383
anatom_site_general        2162
anatom_site_special       11183
benign_malignant           2678
concomitant_biopsy            0
diagnosis                     0
diagnosis_1                   0
diagnosis_2                   0
diagnosis_3                 180
diagnosis_confirm_type        0
lesion_id                     0
melanocytic                   0
sex                         343
image_name                    0
mapped_label                  0
dtype: int64
In [10]:
# Fill missing values in categorical columns with 'Unknown'
metadata['anatom_site_general'] = metadata['anatom_site_general'].fillna('Unknown')
metadata['anatom_site_special'] = metadata['anatom_site_special'].fillna('Unknown')
metadata['benign_malignant'] = metadata['benign_malignant'].fillna('Unknown')


# Fill missing values in numerical columns like 'age_approx' with the median
metadata['age_approx'] = metadata['age_approx'].fillna(metadata['age_approx'].median())

# Fill missing values in categorical columns with 'Unknown' or the mode
metadata['diagnosis_3'] = metadata['diagnosis_3'].fillna('Unknown')
metadata['sex'] = metadata['sex'].fillna(metadata['sex'].mode()[0])
In [11]:
print(metadata.isnull().sum())  # Check if any columns still have missing values

isic_id                   0
age_approx                0
anatom_site_general       0
anatom_site_special       0
benign_malignant          0
concomitant_biopsy        0
diagnosis                 0
diagnosis_1               0
diagnosis_2               0
diagnosis_3               0
diagnosis_confirm_type    0
lesion_id                 0
melanocytic               0
sex                       0
image_name                0
mapped_label              0
dtype: int64
In [12]:
print(metadata.head())  # Check the first few rows of the DataFrame

        isic_id  age_approx anatom_site_general anatom_site_special  \
0  ISIC_0024306        45.0             Unknown             Unknown   
1  ISIC_0024307        50.0     lower extremity             Unknown   
2  ISIC_0024308        55.0             Unknown             Unknown   
3  ISIC_0024309        40.0             Unknown             Unknown   
4  ISIC_0024310        60.0      anterior torso             Unknown   

  benign_malignant  concomitant_biopsy diagnosis diagnosis_1  \
0           benign               False     nevus      Benign   
1           benign               False     nevus      Benign   
2           benign               False     nevus      Benign   
3           benign               False     nevus      Benign   
4        malignant                True  melanoma   Malignant   

                                       diagnosis_2    diagnosis_3  \
0                Benign melanocytic proliferations          Nevus   
1                Benign melanocytic proliferations          Nevus   
2                Benign melanocytic proliferations          Nevus   
3                Benign melanocytic proliferations          Nevus   
4  Malignant melanocytic proliferations (Melanoma)  Melanoma, NOS   

             diagnosis_confirm_type   lesion_id  melanocytic     sex  \
0  serial imaging showing no change  IL_7252831         True    male   
1  serial imaging showing no change  IL_6125741         True    male   
2  serial imaging showing no change  IL_3692653         True  female   
3  serial imaging showing no change  IL_0959663         True    male   
4                    histopathology  IL_8194852         True    male   

         image_name  mapped_label  
0  ISIC_0024306.jpg             0  
1  ISIC_0024307.jpg             0  
2  ISIC_0024308.jpg             0  
3  ISIC_0024309.jpg             0  
4  ISIC_0024310.jpg             1
In [13]:
label_encoder = LabelEncoder()
metadata['sex'] = label_encoder.fit_transform(metadata['sex'])

CustomImageDataset: A PyTorch Dataset for Loading and Transforming Images¶

In [14]:
class CustomImageDataset(Dataset):
    def __init__(self, metadata_df, root_dir, transform=None):
        """
        Args:
            metadata_df (DataFrame): DataFrame containing image metadata.
            root_dir (str): Directory with all the images.
            transform (callable, optional): Optional transform to be applied on a sample.
        """
        self.metadata = metadata_df  # The DataFrame containing metadata
        self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1)  # Map diagnosis to numeric labels (axis=1 applies to rows)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        # Return the total number of images in the dataset
        return len(self.metadata)

    def __getitem__(self, idx):
        # Get image name (ensure you append the file extension like '.jpg')
        img_name = self.metadata.iloc[idx, 0] + ".jpg"  # Assuming the first column has image names without extension
        img_path = os.path.join(self.root_dir, img_name)  # Get full image path

        # Check if the file exists (optional but good practice)
        if not os.path.exists(img_path):
            raise FileNotFoundError(f"Image {img_name} not found in {self.root_dir}")

        # Load the image
        image = Image.open(img_path).convert("RGB")  # Open and convert to RGB if not already

        # Get the corresponding label (mapped label from the DataFrame)
        label = self.metadata.iloc[idx]["mapped_label"]

        # Apply transformations if provided (e.g., resize, normalize, etc.)
        if self.transform:
            image = self.transform(image)

        return image, label  # Return the image and its label

run the model on the device CPU/GPU¶

In [15]:
print(torch.cuda.is_available())  
print(torch.cuda.get_device_name(0))  
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

True
NVIDIA GeForce RTX 4080 SUPER

Setting Up ResNet-50 with Class Weights and Optimization¶

In [16]:
# feature_extract is a boolean that defines if we are finetuning or feature extracting. 
# If feature_extract = False, the model is finetuned and all model parameters are updated. 
# If feature_extract = True, only the last layer parameters are updated, the others remain fixed.
def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False
In [17]:
class_weights = compute_class_weight(
    class_weight="balanced",  # Automatically balance class weights
    classes=np.array([0, 1, 2]),  # Specify class labels explicitly
    y=metadata["mapped_label"].values  # Convert the 'mapped_label' column to a numpy array
)
class_weights = torch.tensor(class_weights, dtype=torch.float32).to(device)
print("Class Weights:", class_weights)
criterion = torch.nn.CrossEntropyLoss(weight=class_weights)  # Loss function for multi-class classification

#criterion = nn.CrossEntropyLoss().to(device)
# Load ResNet-50 model with updated weights parameter
#model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
model = models.resnet50(weights=ResNet50_Weights.DEFAULT)

set_parameter_requires_grad(model, feature_extracting=False)
num_ftrs = model.fc.in_features
# Modify the final layer for 3 classes: 'nevus', 'melanoma', 'other lesion'
model.fc = torch.nn.Linear(num_ftrs, 3)


# Initialize optimizer
# optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4)

Class Weights: tensor([0.4149, 2.9936, 3.9067], device='cuda:0')
In [18]:
# Split the data into training and validation sets (80% training, 20% validation)
train_metadata, val_metadata = train_test_split(metadata, test_size=0.2, random_state=42)

# Check the split sizes
print(f"Training set size: {len(train_metadata)}")
print(f"Validation set size: {len(val_metadata)}")

Training set size: 9376
Validation set size: 2344
In [19]:
# Initialize the CustomImageDataset with the DataFrame
train_dataset = CustomImageDataset(metadata_df=train_metadata, root_dir=root_dir, transform=transform)
val_dataset = CustomImageDataset(metadata_df=val_metadata, root_dir=root_dir, transform=transform)
In [20]:
# Function to apply Grad-CAM
class GradCAM:
    def __init__(self, model, target_layer):
        self.model = model
        self.target_layer = target_layer
        self.gradients = None
        self.activation = None

        # Hook to get gradients
        self.target_layer.register_forward_hook(self.save_activation)
        self.target_layer.register_backward_hook(self.save_gradient)

    def save_activation(self, module, input, output):
        self.activation = output

    def save_gradient(self, module, grad_input, grad_output):
        self.gradients = grad_output[0]

    def generate_cam(self, class_idx):
        grad_val = self.gradients.cpu().data.numpy()
        act_val = self.activation.cpu().data.numpy()

        weights = np.mean(grad_val, axis=(2, 3), keepdims=True)
        cam = np.sum(weights * act_val, axis=1)

        # Normalize and resize CAM
        cam = np.maximum(cam, 0)  # ReLU
        cam = cam[0]  # Get first item
        cam = cv2.resize(cam, (224, 224))  # Resize to match image

        # Normalize CAM
        cam = cam - np.min(cam)
        cam = cam / np.max(cam)

        return cam

Training the Model with Early Stopping and Validation¶

In [21]:
# Define a function for the training loop
def train_one_epoch(model, train_loader, criterion, optimizer, device, epoch, clip_grad_norm=None):
    model.train()
    running_loss = 0.0
    correct_predictions = 0
    total_predictions = 0
    start_time = time.time()  # Track epoch time
    
    for i, (inputs, labels) in enumerate(train_loader):
        inputs, labels = inputs.to(device), labels.to(device)

        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        
        # Gradient clipping (if applicable)
        if clip_grad_norm:
            torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad_norm)

        optimizer.step()

        running_loss += loss.item()
        _, predicted = torch.max(outputs, 1)
        correct_predictions += (predicted == labels).sum().item()
        total_predictions += labels.size(0)

        # Log learning rate every 100 iterations 
        if (i + 1) % 100 == 0:
            print(f"Epoch {epoch}, Iteration {i + 1}/{len(train_loader)} - Learning Rate: {optimizer.param_groups[0]['lr']:.6f}")


    # Calculate average loss and accuracy
    train_loss = running_loss / len(train_loader)
    train_accuracy = 100 * correct_predictions / total_predictions
    
    # Print epoch time
    epoch_time = time.time() - start_time
    print(f"Epoch {epoch} completed in {epoch_time:.2f} seconds")

    return train_loss, train_accuracy

# Define a function for the validation loop
def validate_model(model, val_loader, criterion, device, num_examples=5):
    model.eval()
    val_loss = 0.0
    val_correct_predictions = 0
    val_total_predictions = 0

    # Store misclassified images, true labels, and predicted labels
    misclassified_images = []
    true_labels = []
    predicted_labels = []

    with torch.no_grad():
        for inputs, labels in val_loader:
            inputs, labels = inputs.to(device), labels.to(device)
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            val_loss += loss.item()

            _, predicted = torch.max(outputs, 1)
            val_correct_predictions += (predicted == labels).sum().item()
            val_total_predictions += labels.size(0)

            # Find misclassified images
            misclassified_mask = predicted != labels
            misclassified_images.extend(inputs[misclassified_mask].cpu())
            true_labels.extend(labels[misclassified_mask].cpu())
            predicted_labels.extend(predicted[misclassified_mask].cpu())

    val_loss /= len(val_loader)
    val_accuracy = 100 * val_correct_predictions / val_total_predictions

    return val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels
In [22]:
# Initialize lists to store loss and accuracy
train_losses = []
val_losses = []
train_accuracies = []
val_accuracies = []

# Set up the DataLoader
batch_size = 32
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)

# Set early stopping parameters
patience = 4  # Number of epochs to wait for improvement
best_val_loss = float('inf')
best_Accuracy = float('inf')
epochs_without_improvement = 0

num_epochs = 15  # Set the number of epochs
# Set up device
model.to(device)

# Store misclassified images at the end
final_misclassified_images = []
final_true_labels = []
final_predicted_labels = []

# Training and validation loop
for epoch in range(num_epochs):
    train_loss, train_accuracy = train_one_epoch(model, train_loader, criterion, optimizer, device, epoch)
    train_losses.append(train_loss)
    train_accuracies.append(train_accuracy)

    print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {train_loss:.4f}, Accuracy: {train_accuracy:.2f}%")

    val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels = validate_model(model, val_loader, criterion, device)
    val_losses.append(val_loss)
    val_accuracies.append(val_accuracy)

    # Save the last batch of misclassified images
    final_misclassified_images = misclassified_images
    final_true_labels = true_labels
    final_predicted_labels = predicted_labels

    print(f"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")

    # Early stopping check
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_accuracy = val_accuracy
        epochs_without_improvement = 0
        torch.save(model.state_dict(), 'resnet50_custom_best.pth')
        torch.save(optimizer.state_dict(), 'resnet50_optimizer_best.pth')
        print("Best Model saved")
    else:
        epochs_without_improvement += 1
        if epochs_without_improvement >= patience:
            print("Early stopping triggered!")
            break

# Print final results
print(f"Finished Training and Validation, Best Model Accuracy: {best_accuracy:.2f}%, Loss: {best_val_loss:.4f}")

Epoch 0, Iteration 100/293 - Learning Rate: 0.001000
Epoch 0, Iteration 200/293 - Learning Rate: 0.001000
Epoch 0 completed in 78.34 seconds
Epoch [1/15], Loss: 0.8123, Accuracy: 66.08%
Validation Loss: 0.6486, Validation Accuracy: 69.28%
Best Model saved
Epoch 1, Iteration 100/293 - Learning Rate: 0.001000
Epoch 1, Iteration 200/293 - Learning Rate: 0.001000
Epoch 1 completed in 77.71 seconds
Epoch [2/15], Loss: 0.5882, Accuracy: 71.14%
Validation Loss: 0.5803, Validation Accuracy: 75.43%
Best Model saved
Epoch 2, Iteration 100/293 - Learning Rate: 0.001000
Epoch 2, Iteration 200/293 - Learning Rate: 0.001000
Epoch 2 completed in 78.60 seconds
Epoch [3/15], Loss: 0.5274, Accuracy: 74.43%
Validation Loss: 0.4927, Validation Accuracy: 75.64%
Best Model saved
Epoch 3, Iteration 100/293 - Learning Rate: 0.001000
Epoch 3, Iteration 200/293 - Learning Rate: 0.001000
Epoch 3 completed in 77.59 seconds
Epoch [4/15], Loss: 0.4833, Accuracy: 75.17%
Validation Loss: 0.4991, Validation Accuracy: 77.13%
Epoch 4, Iteration 100/293 - Learning Rate: 0.001000
Epoch 4, Iteration 200/293 - Learning Rate: 0.001000
Epoch 4 completed in 78.78 seconds
Epoch [5/15], Loss: 0.4552, Accuracy: 77.60%
Validation Loss: 0.4733, Validation Accuracy: 76.15%
Best Model saved
Epoch 5, Iteration 100/293 - Learning Rate: 0.001000
Epoch 5, Iteration 200/293 - Learning Rate: 0.001000
Epoch 5 completed in 78.45 seconds
Epoch [6/15], Loss: 0.4032, Accuracy: 79.24%
Validation Loss: 0.4667, Validation Accuracy: 79.05%
Best Model saved
Epoch 6, Iteration 100/293 - Learning Rate: 0.001000
Epoch 6, Iteration 200/293 - Learning Rate: 0.001000
Epoch 6 completed in 78.85 seconds
Epoch [7/15], Loss: 0.3744, Accuracy: 81.24%
Validation Loss: 0.4554, Validation Accuracy: 78.97%
Best Model saved
Epoch 7, Iteration 100/293 - Learning Rate: 0.001000
Epoch 7, Iteration 200/293 - Learning Rate: 0.001000
Epoch 7 completed in 78.86 seconds
Epoch [8/15], Loss: 0.3491, Accuracy: 81.99%
Validation Loss: 0.4422, Validation Accuracy: 81.19%
Best Model saved
Epoch 8, Iteration 100/293 - Learning Rate: 0.001000
Epoch 8, Iteration 200/293 - Learning Rate: 0.001000
Epoch 8 completed in 79.05 seconds
Epoch [9/15], Loss: 0.3330, Accuracy: 82.81%
Validation Loss: 0.4419, Validation Accuracy: 81.53%
Best Model saved
Epoch 9, Iteration 100/293 - Learning Rate: 0.001000
Epoch 9, Iteration 200/293 - Learning Rate: 0.001000
Epoch 9 completed in 78.79 seconds
Epoch [10/15], Loss: 0.3076, Accuracy: 84.33%
Validation Loss: 0.3995, Validation Accuracy: 82.64%
Best Model saved
Epoch 10, Iteration 100/293 - Learning Rate: 0.001000
Epoch 10, Iteration 200/293 - Learning Rate: 0.001000
Epoch 10 completed in 79.41 seconds
Epoch [11/15], Loss: 0.2788, Accuracy: 85.74%
Validation Loss: 0.4378, Validation Accuracy: 76.96%
Epoch 11, Iteration 100/293 - Learning Rate: 0.001000
Epoch 11, Iteration 200/293 - Learning Rate: 0.001000
Epoch 11 completed in 78.71 seconds
Epoch [12/15], Loss: 0.2547, Accuracy: 86.39%
Validation Loss: 0.4325, Validation Accuracy: 82.72%
Epoch 12, Iteration 100/293 - Learning Rate: 0.001000
Epoch 12, Iteration 200/293 - Learning Rate: 0.001000
Epoch 12 completed in 79.36 seconds
Epoch [13/15], Loss: 0.2375, Accuracy: 87.17%
Validation Loss: 0.4501, Validation Accuracy: 80.38%
Epoch 13, Iteration 100/293 - Learning Rate: 0.001000
Epoch 13, Iteration 200/293 - Learning Rate: 0.001000
Epoch 13 completed in 78.73 seconds
Epoch [14/15], Loss: 0.2330, Accuracy: 87.95%
Validation Loss: 0.5326, Validation Accuracy: 84.68%
Early stopping triggered!
Finished Training and Validation, Best Model Accuracy: 82.64%, Loss: 0.3995

Visualizing Model Performance: Loss & Accuracy Trends¶

In [23]:
# Plot Training & Validation Loss
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_losses) + 1), train_losses, label="Training Loss", marker='o')
plt.plot(range(1, len(val_losses) + 1), val_losses, label="Validation Loss", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Training & Validation Loss")
plt.legend()
plt.grid()
plt.show()

# Plot Training & Validation Accuracy
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_accuracies) + 1), train_accuracies, label="Training Accuracy", marker='o')
plt.plot(range(1, len(val_accuracies) + 1), val_accuracies, label="Validation Accuracy", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Accuracy (%)")
plt.title("Training & Validation Accuracy")
plt.legend()
plt.grid()
plt.show()
No description has been provided for this image
No description has been provided for this image

Confusion Matrix and Misclassified Images Analysis¶

In [24]:
# Define a mapping for label names
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}

model.eval()
y_true = []
y_pred = []

with torch.no_grad():
    for images, labels in val_loader:
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)
        predictions = outputs.max(1)[1]  # Get predicted class index

        y_true.extend(labels.cpu().numpy())
        y_pred.extend(predictions.cpu().numpy())

# Compute confusion matrix 
confusion_mtx = confusion_matrix(y_true, y_pred)

# Function to plot confusion matrix with COUNT values
def plot_confusion_matrix(cm, classes, normalize=False, title="Confusion Matrix", cmap=plt.cm.Blues):
    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]  # Normalize by row sum

    # Define threshold for color contrast
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        color = "green" if i == j else "red"  # Green for correct, Red for incorrect
        plt.text(j, i, f"{cm[i, j]:.2f}" if normalize else f"{cm[i, j]}",
                 horizontalalignment="center",
                 color=color, fontsize=12, fontweight="bold")

    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.tight_layout()
    plt.show()

# Plot confusion matrix as COUNT instead of percentage
plot_confusion_matrix(confusion_mtx, list(label_mapping.values()), normalize=False, title="Confusion Matrix")


if len(final_misclassified_images) > 0:
    print(f"Showing {min(5, len(final_misclassified_images))} misclassified images from final validation:")
    for i in range(min(5, len(final_misclassified_images))):
        image = final_misclassified_images[i]
        true_label = label_mapping[final_true_labels[i].item()]
        predicted_label = label_mapping[final_predicted_labels[i].item()]

        # Convert tensor to numpy and denormalize
        image = image.permute(1, 2, 0).numpy() 
        if image.min() < 0 or image.max() > 1: 
            image = (image - image.min()) / (image.max() - image.min())

        # Plot the image
        plt.figure(figsize=(5, 5))
        plt.imshow(image)
        plt.title(f"True: {true_label}, Predicted: {predicted_label}")
        plt.axis("off")
        plt.show()
No description has been provided for this image 
Showing 5 misclassified images from final validation:
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image

Our model's main misclassifications occur when melanoma is predicted as nevus and when other lesions are mistaken for nevus.
Nonetheless, we achieved an 86% prediction accuracy and a validation loss of 0.3306, which is quite decent.

Grad-CAM Visualization for Model Predictions¶

In [25]:
# Path to the directory containing images
image_dir = r"E:\afeka\finalProject\JupyterLab\images_ToTest\images"

# Get all image files in the directory
image_files = [f for f in os.listdir(image_dir) if f.endswith(('.png', '.jpg', '.jpeg'))]

# Set model to evaluation mode
model.eval()

# Define confidence threshold (adjust as needed)
confidence_threshold = 70  # Threshold in percentage 

# Class labels
class_labels = ['nevus', 'melanoma', 'other lesion']

# Choose the layer for ResNet (last conv layer in layer4)
target_layer = model.layer4[-1] 

# Initialize Grad-CAM
grad_cam = GradCAM(model=model, target_layer=target_layer)

# Iterate through all images
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    
    # Load and preprocess the image
    image = Image.open(image_path)
    input_tensor = transform(image).unsqueeze(0).to(device)  # Add batch dimension and move to device
    input_tensor.requires_grad_(True)  # Enable gradient computation

    # Forward pass
    output = model(input_tensor)
    probabilities = torch.nn.functional.softmax(output, dim=1)  # Get softmax probabilities
    confidence, predicted = torch.max(probabilities, 1)  # Get max probability and index

    # Perform backward pass to compute gradients
    model.zero_grad()  # Clear existing gradients
    output[0, predicted.item()].backward()  # Backward pass for the predicted class

    predicted_class = class_labels[predicted.item()]
    confidence_percentage = confidence.item() * 100  # Convert to percentage

    # Extract image ID from filename 
    image_id = os.path.splitext(image_file)[0]  

    # Generate the Grad-CAM heatmap
    cam = grad_cam.generate_cam(predicted.item())  # Pass the predicted class index

    # Convert original image to NumPy array
    image_np = np.array(image.resize((224, 224)))

    # Squeeze the CAM to ensure it's 2D
    cam = np.squeeze(cam)

    # Normalize and convert cam to uint8
    cam = cam - np.min(cam)  # Shift values to positive range
    cam = cam / np.max(cam)  # Normalize to [0,1]
    cam = np.uint8(255 * cam)  # Scale to [0,255] and convert to uint8

    # Overlay heatmap on image
    heatmap = cv2.applyColorMap(cam, cv2.COLORMAP_JET)
    superimposed_img = cv2.addWeighted(image_np, 0.6, heatmap, 0.4, 0)

    # Print results with percentage confidence and display Grad-CAM
    if confidence_percentage < confidence_threshold:
        print(f"Image ID: {image_id}, Prediction Uncertain (Confidence: {confidence_percentage:.2f}%)")
    else:
        print(f"Image ID: {image_id}, Predicted Outcome: {predicted_class} (Confidence: {confidence_percentage:.2f}%)")

    # Display the original and Grad-CAM result
    plt.figure(figsize=(10, 5))
    plt.subplot(1, 2, 1)
    plt.imshow(image_np)
    plt.title(f"Original Image: {image_id}")

    plt.subplot(1, 2, 2)
    plt.imshow(superimposed_img)
    plt.title("Grad-CAM Heatmap")
    

    plt.show()

C:\Users\mantr\AppData\Local\Programs\Python\Python312\Lib\site-packages\torch\nn\modules\module.py:1830: FutureWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.
  self._maybe_warn_non_full_backward_hook(args, result, grad_fn)

Image ID: Car_Image, Prediction Uncertain (Confidence: 50.73%)
No description has been provided for this image 
Image ID: ISIC_1871644, Predicted Outcome: nevus (Confidence: 99.92%)
No description has been provided for this image 
Image ID: ISIC_2175344, Predicted Outcome: other lesion (Confidence: 93.11%)
No description has been provided for this image 
Image ID: ISIC_2310644, Predicted Outcome: melanoma (Confidence: 87.87%)
No description has been provided for this image 
Image ID: ISIC_5178159, Prediction Uncertain (Confidence: 53.62%)
No description has been provided for this image 
Image ID: ISIC_5562564, Predicted Outcome: nevus (Confidence: 99.33%)
No description has been provided for this image 
Image ID: ISIC_6377613, Predicted Outcome: other lesion (Confidence: 99.89%)
No description has been provided for this image 
Image ID: ISIC_6821316, Predicted Outcome: melanoma (Confidence: 99.98%)
No description has been provided for this image 
Image ID: ISIC_7522225, Predicted Outcome: nevus (Confidence: 99.40%)
No description has been provided for this image 
Image ID: ISIC_9105056, Prediction Uncertain (Confidence: 65.43%)
No description has been provided for this image 
Image ID: ISIC_9911782, Predicted Outcome: nevus (Confidence: 99.99%)
No description has been provided for this image 
Image ID: ISIC_9957003, Predicted Outcome: melanoma (Confidence: 95.09%)
No description has been provided for this image
In [ ]: